# -*- coding: utf-8 -*-
"""
中性策略框架 | 邢不行 | 2024分享会
author: 邢不行
微信: xbx6660
"""
import os
import time
import traceback
import warnings
warnings.filterwarnings('ignore')

import numpy as np
import pandas as pd
from config import utc_offset, select_coin_path, glob_var, order_path, data_center_path, data_path
from api.market import fetch_binance_ticker_data, reset_leverage
from api.position import replenish_bnb, get_spot_position_and_equity, bn_transfer_spot_to_u, get_position_df, get_equity
from api.trade import transfer_bnb
from utils.shift_functions import calc_curve, transfer_curve_for_shift_cfg, calc_strategy_factors, select_all_offset_strategy, get_select_coin_for_strategy
from utils.commons import retry_wrapper
from utils.dingding import send_wechat_work_msg


def calc_rank(df, factor_column='因子', ascending=True):
    """
    计算因子排名
    :param df:              原数据
    :param factor_column:   需要计算排名的因子名称
    :param ascending:       计算排名顺序，True：从小到达排序；False：从大到小排序
    :return:
    """
    # 计算因子的分组排名
    df['rank'] = df.groupby('candle_begin_time')[factor_column].rank(method='min', ascending=ascending)
    df['rank_max'] = df.groupby('candle_begin_time')['rank'].transform('max')
    # 根据时间和因子排名
    df.sort_values(by=['candle_begin_time', 'rank'], inplace=True)
    # 重新计算一下总币数
    df['总币数'] = df.groupby('candle_begin_time')['symbol'].transform('size')

    return df


def select_long_and_short_coin(long_df, short_df, long_select_coin_num, short_select_coin_num, long_factor='因子', short_factor='因子'):
    """
    选币
    :param long_df:                 多头选币的df
    :param short_df:                空头选币的df
    :param long_select_coin_num:    多头选币数量
    :param short_select_coin_num:   空头选币数量
    :param long_factor:             做多因子名称
    :param short_factor:            做空因子名称
    :return:
    """
    # ===做多选币，因子值相同时全选
    long_df = calc_rank(long_df, factor_column=long_factor, ascending=True)
    # 百分比选币模式
    if int(long_select_coin_num) == 0:
        long_df = long_df[long_df['rank'] <= long_df['总币数'] * long_select_coin_num]
    else:
        long_df = long_df[long_df['rank'] <= long_select_coin_num]
    long_df['方向'] = 1

    # ===做空选币
    short_df = calc_rank(short_df, factor_column=short_factor, ascending=False)
    if short_select_coin_num == 'long_nums':  # 如果参数是long_nums，则空头与多头的选币数量保持一致
        # 获取到多头的选币数量并整理数据
        long_select_num = long_df.groupby('candle_begin_time')['symbol'].size().to_frame()
        long_select_num = long_select_num.rename(columns={'symbol': '多头数量'}).reset_index()
        # 将多头选币数量整理到short_df
        short_df = short_df.merge(long_select_num, on='candle_begin_time', how='left')
        # 使用多头数量对空头数据进行选币
        short_df = short_df[short_df['rank'] <= short_df['多头数量']]
        del short_df['多头数量']
    else:
        # 百分比选币
        if int(short_select_coin_num) == 0:
            short_df = short_df[short_df['rank'] <= short_df['总币数'] * short_select_coin_num]
        # 固定数量选币
        else:
            short_df = short_df[short_df['rank'] <= short_select_coin_num]
    short_df['方向'] = -1

    # ===整理数据
    df = pd.concat([long_df, short_df], ignore_index=True)   # 将做多和做空的币种数据合并
    df.sort_values(by=['candle_begin_time', '方向'], ascending=[True, False], inplace=True)
    df.reset_index(drop=True, inplace=True)
    del df['总币数'], df['rank_max']

    return df


def cal_factors(df, all_factor_list, run_time=None, hold_period=None, all_filter_list=[], save_cols=[]):
    # 遍历每个因子，计算每个因子的数据
    for factor, if_reverse, parameter_list, weight in all_factor_list:
        factor_name = f'{factor}_{str(parameter_list)}'
        if factor_name in df.columns:
            continue
        _cls = __import__('factors.%s' % factor, fromlist=('',))
        df = getattr(_cls, 'signal')(df, int(parameter_list), factor_name)

    # 如果配置了过滤因子，解析出过滤因子的信息，并进行计算
    if len(all_filter_list) > 0:
        for factor, parameter_list in all_filter_list:
            factor_name = f'{factor}_{str(parameter_list)}'
            if factor_name in df.columns:
                continue
            _cls = __import__('factors.%s' % factor, fromlist=('',))
            df = getattr(_cls, 'signal')(df, int(parameter_list), factor_name)

    # 只保留最近的数据
    if run_time and hold_period:
        df = df[df['candle_begin_time'] >= (run_time - pd.to_timedelta(hold_period) - pd.Timedelta(hours=utc_offset))]
    # 只保留需要的字段
    df = df[['candle_begin_time', 'symbol', 'symbol_type', 'close'] + save_cols]

    return df


# =====策略相关函数
def del_insufficient_data(symbol_candle_data):
    """
    删除数据长度不足的币种信息

    :param symbol_candle_data:
    :return
    """
    # ===删除成交量为0的线数据、k线数不足的币种
    symbol_list = list(symbol_candle_data.keys())
    for symbol in symbol_list:
        # 删除空的数据
        if symbol_candle_data[symbol] is None or symbol_candle_data[symbol].empty:
            del symbol_candle_data[symbol]
            continue
        # 删除该币种成交量=0的k线
        symbol_candle_data[symbol] = symbol_candle_data[symbol][symbol_candle_data[symbol]['volume'] > 0]

    return symbol_candle_data


def transfer_swap(select_coin, df_swap, special_symbol_dict):
    """
    将现货中的数据替换成合约数据，主要替换：close
    :param select_coin:     选币数据
    :param df_swap:         合约数据
    :param special_symbol_dict: 特殊币种对应
    :return:
    """
    # 反转k,v的对应关系。现货对应合约，转换 ，合约对应现货
    special_symbol_dict = {v: k for k, v in special_symbol_dict.items()}
    # 整理币种名称，使合约和现货的币种名称相同
    df_swap['symbol_spot'] = df_swap['symbol'].apply(lambda x: special_symbol_dict.get(x.replace('USDT', ''), (x.split('1000')[1] if '1000' in x else x).replace('USDT', ''))+'USDT')
    # 设置指定字段
    df_swap = df_swap[['candle_begin_time', 'symbol', 'symbol_type', 'close', 'tag', 'symbol_spot']]

    # 将选币结果中是否含有合约的数据筛选出来
    not_swap_df = select_coin[select_coin['tag'] != 'HasSwap']
    has_swap_df = select_coin[select_coin['tag'] == 'HasSwap']  # 对含有合约的现货数据进行处理

    # 设置需要替换的字段
    other_cols = list(set(select_coin.columns) - set(df_swap.columns))
    has_swap_df = pd.merge(left=has_swap_df[['candle_begin_time', 'symbol'] + other_cols], right=df_swap,
                           left_on=['candle_begin_time', 'symbol'], right_on=['candle_begin_time', 'symbol_spot'],
                           how='left', suffixes=('', '_swap'))
    has_swap_df['symbol'] = has_swap_df['symbol_swap']
    del has_swap_df['symbol_swap'], has_swap_df['symbol_spot']

    # 将拆分的选币数据，合并回去
    select_coin = pd.concat([not_swap_df, has_swap_df], axis=0)
    select_coin.sort_values(['candle_begin_time', '方向'], inplace=True)
    select_coin.reset_index(inplace=True, drop=True)

    return select_coin


# 选币数据整理 & 选币
def cal_factor_and_select_coin(symbol_spot_candle_data, symbol_swap_candle_data, run_time, account_info, special_symbol_dict={}):
    """
    选币数据整理 & 选币
    :param symbol_spot_candle_data:      1h的k线数据
    :param symbol_swap_candle_data:      1h的k线数据
    :param run_time:                    当前运行的时间
    :param account_info:                账户配置信息
    :param special_symbol_dict:         特殊现货与合约的对应关系
    :return:

         candle_begin_time    symbol  close  方向  offset              key
    0  2022-10-28 09:00:00   EOSUSDT  1.140   1       3  c_factor1_6H_3H
    1  2022-10-28 09:00:00  LINKUSDT  6.940  -1       3  c_factor1_6H_3H
    2  2022-10-28 10:00:00   EOSUSDT  1.127   1       4  c_factor1_6H_4H
    3  2022-10-28 10:00:00  LINKUSDT  6.861  -1       4  c_factor1_6H_4H
    4  2022-10-28 11:00:00   EOSUSDT  1.136   1       5  c_factor1_6H_5H
    5  2022-10-28 11:00:00  LINKUSDT  6.893  -1       5  c_factor1_6H_5H
    6  2022-10-28 12:00:00   EOSUSDT  1.142   1       0  c_factor1_6H_0H
    7  2022-10-28 12:00:00  LINKUSDT  6.899  -1       0  c_factor1_6H_0H
    8  2022-10-28 13:00:00   EOSUSDT  1.139   1       1  c_factor1_6H_1H
    9  2022-10-28 13:00:00  LINKUSDT  6.918  -1       1  c_factor1_6H_1H
    10 2022-10-28 14:00:00   EOSUSDT  1.142   1       2  c_factor1_6H_2H
    11 2022-10-28 14:00:00  LINKUSDT  6.926  -1       2  c_factor1_6H_2H
    """
    # ===删除一些数据不全的币种
    symbol_spot_candle_data = del_insufficient_data(symbol_spot_candle_data)
    symbol_swap_candle_data = del_insufficient_data(symbol_swap_candle_data)

    all_strategy_df = []
    for stg in account_info['strategy_list']:
        # 判断是不是轮动实盘
        if stg['strategy'].startswith('Shift'):
            _s_time = time.time()
            all_df = calc_shift_select_coin(stg, symbol_spot_candle_data, symbol_swap_candle_data, run_time, special_symbol_dict, account_info)
            print('calc_shift_select_coin:', time.time() - _s_time)
        else:
            all_df = calc_strategy_select_coin(stg, symbol_spot_candle_data, symbol_swap_candle_data, run_time, special_symbol_dict, account_info)
        all_strategy_df.append(all_df)

    all_strategy_df = pd.concat(all_strategy_df, ignore_index=True)

    # ===返回选币结果
    return all_strategy_df


def get_swap_tag(df, special_symbol_dict, min_kline_num):
    """
    在数据中加入tag列，表明现货是否拥有对应的合约（主要用于历史数据判断，最新数据不用这么费劲儿）
    :param df:
    :param special_symbol_dict: 特殊现货对应合约关系
    :param min_kline_num: 最少上市多久
    :return:
    """
    swap_path = os.path.join(data_center_path, 'kline/swap')
    symbol = df.iloc[0]['symbol'].replace('USDT', '')  # 只保留币种名称，不含结算币种
    if df.iloc[0]['symbol_type'] == 'spot':
        # 现货的资金费率为0
        df['fundingRate'] = 0
        # 拼接出该币种合约数据的路径
        _symbol = special_symbol_dict.get(symbol, symbol)  # 兼容币种别名的
        _swap_file = os.path.join(swap_path, _symbol + 'USDT.csv')
        _swap_file2 = os.path.join(swap_path, '1000' + _symbol +'USDT.csv')
        # 如果存在合约数据的，在合约的上时间范围内标记HasSwap
        if os.path.exists(_swap_file) or os.path.exists(_swap_file2):
            try:
                _swap_df = pd.read_csv(_swap_file, encoding='gbk', parse_dates=['candle_begin_time'])
            except:
                _swap_df = pd.read_csv(_swap_file2, encoding='gbk', parse_dates=['candle_begin_time'])
            cond1 = df['candle_begin_time'] > _swap_df.iloc[0]['candle_begin_time'] + pd.to_timedelta('1H') * (min_kline_num + 24)
            cond2 = df['candle_begin_time'] <= _swap_df.iloc[-1]['candle_begin_time']
            df.loc[cond1 & cond2, 'tag'] = 'HasSwap'

        # 合约有改名的情况 ========================================
        if _symbol != symbol:  # 兼容币种改名。LUNA -> LUNA2。2022-5-12之前还是LUNA，2022-9-10改名为LUNA2
            _swap_file = os.path.join(swap_path, symbol + 'USDT.csv')
            _swap_file2 = os.path.join(swap_path, '1000' + symbol +'USDT.csv')
            # 如果存在合约数据的，在合约的上时间范围内标记HasSwap
            if os.path.exists(_swap_file) or os.path.exists(_swap_file2):
                try:
                    _swap_df = pd.read_csv(_swap_file, encoding='gbk', parse_dates=['candle_begin_time'])
                except:
                    _swap_df = pd.read_csv(_swap_file2, encoding='gbk', parse_dates=['candle_begin_time'])
                cond1 = df['candle_begin_time'] > _swap_df.iloc[0]['candle_begin_time'] + pd.to_timedelta('1H') * (min_kline_num + 24)
                cond2 = df['candle_begin_time'] <= _swap_df.iloc[-1]['candle_begin_time']
                df.loc[cond1 & cond2, 'tag'] = 'HasSwap'
    else:
        df['tag'] = None

    return df


def calc_strategy_select_coin(stg, symbol_spot_candle_data, symbol_swap_candle_data, run_time, special_symbol_dict, account_info):
    """
    计算策略选币
    :param stg:                          策略配置信息
    :param symbol_spot_candle_data:      1h现货k线数据
    :param symbol_swap_candle_data:      1h合约k线数据
    :param run_time:                    当前运行的时间
    :param account_info:                账户配置信息
    :param special_symbol_dict:         特殊现货与合约的对应关系
    :return:
    """
    # 获取当前策略信息
    strategy = __import__(f'strategy.{stg["strategy"]}', fromlist=('',))
    is_use_spot = stg['is_use_spot'] if 'is_use_spot' in stg else strategy.if_use_spot
    # ===整合一下选币因子和过滤因子信息
    # 将策略配置的因子信息，转换成统一格式，方便后续df操作
    feature_list = []
    for factor_name, if_reverse, parameter_list, weight in strategy.factor_list:
        feature_list.append(f'{factor_name}_{str(parameter_list)}')

    # 将策略配置的过滤信息，转换成统一格式，方便后续df操作
    filter_list = []
    for factor_name, parameter_list in strategy.filter_list:
        filter_list.append(f'{factor_name}_{str(parameter_list)}')

    # =合并 factor_column_list 和 filter_column_list 的信息（会去重）
    all_factor_list = list(set(feature_list + filter_list))

    # ===因子计算
    # hold_period的作用是计算完因子之后，获取最近 hold_period 个小时内的数据信息，同时用于offset字段计算使用
    hold_period = int(strategy.hold_period[:-1])
    # 遍历每个币种，计算相关因子数据
    all_df_list = []
    for df in list(symbol_spot_candle_data.copy().values()) + list(symbol_swap_candle_data.copy().values()):
        # 计算因子
        _symbol = df.iloc[0]['symbol'].replace('USDT', '')
        df = cal_factors(df, strategy.factor_list, run_time=run_time, hold_period=strategy.hold_period,
                         all_filter_list=strategy.filter_list, save_cols=all_factor_list)
        if df is None or df.empty:
            continue
        # 追加是否有合约标记
        df = get_swap_tag(df, special_symbol_dict, account_info['min_kline_size'])
        all_df_list.append(df)

    # ===合并并整理所有K线
    all_df = pd.concat(all_df_list, ignore_index=True)

    # 保留一下合约数据
    df_spot = all_df[all_df['symbol_type'] == 'spot']  # 取出现货数据，后续根据配置用来进行过滤和选币
    df_swap = all_df[all_df['symbol_type'] == 'swap']  # 取出合约数据，后续根据配置用来进行过滤和选币，并替换掉现货中的行情数据

    # 判断是否使用了现货数据
    if is_use_spot:  # 使用现货数据，则在现货中进行过滤，并选币
        all_df = df_spot.copy()
    else:  # 使用合约数据，则在合约中进行过滤，并选币
        all_df = df_swap.copy()

    all_df.dropna(subset=all_factor_list, inplace=True)  # 删除选币因子为空的数据
    all_df.sort_values(by=['candle_begin_time', 'symbol'], inplace=True)
    all_df.reset_index(drop=True, inplace=True)

    # ===对因子进行排名
    factor_columns = list({strategy.long_factor, strategy.short_factor})
    save_columns = list(set(factor_columns + filter_list))
    all_df = strategy.calc_factor(all_df)
    all_df = all_df[['candle_begin_time', 'symbol', 'symbol_type', 'tag', 'close'] + save_columns]  # 只保留需要的字段

    # ===前置过滤
    long_df, short_df = strategy.before_filter(all_df)
    if is_use_spot:  # 使用现货数据，则在现货中进行过滤，并选币
        short_df = short_df[short_df['tag'] == 'HasSwap']  # 保留有合约的现货

    # ===选币
    all_df = select_long_and_short_coin(long_df, short_df, strategy.long_select_coin_num,
                                        strategy.short_select_coin_num,
                                        long_factor=strategy.long_factor, short_factor=strategy.short_factor)
    if all_df.empty:  # 如果选币为空，直接返回空的all_df
        return all_df  # 这里不能返回 pd.Dataframe ,因为缺少后面计算的字段

    # ===数据替换
    # 如果是纯多头现货模式，那么就不转换合约数据，只下现货单
    if not account_info['is_pure_long']:
        # 将含有现货的币种，替换掉其中close价格
        all_df = transfer_swap(all_df, df_swap, special_symbol_dict=special_symbol_dict)

    # ===筛选合适的offset
    offset_list = stg['offset_list'] if 'offset_list' in stg else [strategy.offset]
    cal_offset_base_seconds = 3600 * 24 if account_info['is_day_period'] else 3600
    all_df['offset'] = all_df['candle_begin_time'].apply(lambda x: int(((x.to_pydatetime() - pd.to_datetime('2017-01-01')).total_seconds() / cal_offset_base_seconds) % hold_period))
    all_df['offset'] = (all_df['offset'] + 1 + hold_period) % hold_period  # 下单之后，开始持仓offset
    all_df = all_df[all_df['offset'].isin(offset_list)]  # 保留配置中的offset数据

    all_df['strategy'] = stg['strategy']
    all_df['cap_weight'] = stg['cap_weight']
    all_df['offset_num'] = len(offset_list)
    all_df['换仓时间'] = all_df['candle_begin_time'] + pd.to_timedelta(strategy.hold_period)

    return all_df


def calc_shift_select_coin(stg, symbol_spot_candle_data, symbol_swap_candle_data, run_time, special_symbol_dict, account_info):
    """
    计算轮动策略的选币结果

    1）获取轮动策略的子策略
    2）遍历子策略组
    3）计算因子(保留尽可能多的数据)
    4）计算选币
    5）计算资金曲线
    6）根据资金曲线计算轮动策略的因子
    7）选择轮动的子策略
    8）返回轮动的子策略的选币结果

    :param stg:                         策略配置
    :param symbol_spot_candle_data:     1h现货k线数据
    :param symbol_swap_candle_data:     1h合约k线数据
    :param run_time:                    当前运行时间
    :param special_symbol_dict:         特殊别种配置信息
    :param account_info:                当前账号配置信息
    """
    # ===获取轮动策略的子策略
    stg = stg.copy()
    shift_strategy = __import__(f'shift.{stg["strategy"]}', fromlist=('',))
    strategy_dict = shift_strategy.base_strategy_dict
    strategy_list = list(strategy_dict.keys())

    # ===遍历子策略组
    all_select_df = {}
    all_curve_df = []
    for _stg_name in strategy_list:
        # =计算选币
        stg['strategy'] = _stg_name
        _s_time = time.time()
        all_df = calc_strategy_select_coin(stg, symbol_spot_candle_data, symbol_swap_candle_data, None, special_symbol_dict, account_info)
        print('calc_strategy_select_coin:', time.time() - _s_time)
        if all_df.empty:
            continue

        # =数据整理
        # 获取当前策略信息
        strategy = __import__(f'strategy.{_stg_name}', fromlist=('',))
        # 策略对应选币（保留最近一个周期的选币）
        all_select_df[_stg_name] = all_df[all_df['candle_begin_time'] >= (run_time - pd.to_timedelta(strategy.hold_period) - pd.Timedelta(hours=utc_offset))]
        # 获取最新的两个周期时间
        last_time = all_df['candle_begin_time'].drop_duplicates().tail(2)
        last_offset_list = all_df[all_df['candle_begin_time'].isin(last_time)]['offset'].unique()

        # =计算策略资金曲线
        # 遍历所有offset
        for _offset in stg['offset_list']:
            # =计算资金曲线（文件存在时，可以只更新最近3个周期的资金曲线）
            _s_time = time.time()
            curve_df = calc_curve(all_df, strategy, run_time, _offset, last_offset_list)
            print('calc_curve:', time.time() - _s_time)
            if curve_df.empty:  # 资金曲线为空，说明当前offset不需要计算资金曲线，跳过
                continue
            # =周期转换
            _s_time = time.time()
            equity = transfer_curve_for_shift_cfg(curve_df, run_time, strategy_dict, _stg_name, strategy, _offset)
            print('transfer_curve_for_shift_cfg:', time.time() - _s_time)
            all_curve_df.append(equity)

    # ===资金曲线数据合并
    if not all_curve_df:
        return pd.DataFrame()
    base_equity = pd.concat(all_curve_df, ignore_index=True)

    # ===根据资金曲线计算轮动策略的因子
    # 获取因子列名
    factor_cols = []
    for factor_name, if_reverse, parameter_list, weight in shift_strategy.factor_list:
        factor_cols.append(f'{factor_name}_{str(parameter_list)}')

    # 计算因子
    df_list = []
    for symbol in base_equity['symbol'].unique():
        # 因为限制了所有策略的持仓周期一直，所以这里只需要获取任意策略的hold_period即可
        _equity = calc_strategy_factors(base_equity, symbol, shift_strategy, strategy.hold_period, factor_cols)
        df_list.append(_equity)

    # 合并整理数据
    df = pd.concat(df_list, axis=0)
    df = df.dropna(subset=factor_cols, axis=0)  # 删除因子值为空的数据
    df = df.sort_values(['candle_begin_time', 'symbol']).reset_index(drop=True)  # 排序

    # ===选择轮动的子策略
    _s_time = time.time()
    select_strategy_list = select_all_offset_strategy(df, shift_strategy)
    print('select_all_offset_strategy:', time.time() - _s_time)

    # ===获取选中策略的选币
    # 因为限制了所有策略的持仓周期一直，所以这里只需要获取任意策略的hold_period即可
    _s_time = time.time()
    strategy_select_coin_dict = get_select_coin_for_strategy(select_strategy_list, all_select_df, run_time, strategy.hold_period, account_info)
    print('get_select_coin_for_strategy:', time.time() - _s_time)
    # 合并选币
    strategy_select_coin_list = [v['select_coin'] for k, v in strategy_select_coin_dict.items()]
    strategy_select_coin_df = pd.concat(strategy_select_coin_list, ignore_index=True)
    # 重新计算资金占比。若多策略多offset轮动选中，按照各个策略offset占比来计算资金分配。
    strategy_select_coin_df['cap_weight'] *= strategy_select_coin_df.groupby('strategy')['offset'].transform('nunique') / len(stg['offset_list'])
    # 发送轮动状态
    send_shift_status(strategy_select_coin_dict, shift_strategy.stg_name, account_info['wechat_webhook_url'], is_send=True)

    return strategy_select_coin_df


def send_shift_status(strategy_select_coin_dict, stg_name, wechat_webhook_url, is_send=False):
    msg = f'【{stg_name}】轮动状态：\n\n'
    for k, v in strategy_select_coin_dict.items():
        strategy_name = v['status'].split('_eq_')[0]  # 策略名称
        curve_type = v['status'].split('_eq_')[1].replace('资金曲线', '')  # 曲线名称
        offset = v['status'].split('_eq_')[2]  # offset
        msg += f'offset:{offset}，策略:【{strategy_name}】,状态:【{curve_type}】\n'

    print('当前轮动的状态：', msg)
    if is_send:
        send_wechat_work_msg(msg, wechat_webhook_url)


# =====计算实际下单量
def cal_order_amount(select_coin, account_info):
    """
    计算实际下单量

    :param select_coin:             选币结果
    :param account_info:            账户配置信息
    :return:

               当前持仓量   目标持仓量  目标下单份数   实际下单量 交易模式
    AUDIOUSDT         0.0 -2891.524948          -3.0 -2891.524948     建仓
    BANDUSDT        241.1     0.000000           NaN  -241.100000     清仓
    C98USDT        -583.0     0.000000           NaN   583.000000     清仓
    ENJUSDT           0.0  1335.871133           3.0  1335.871133     建仓
    WAVESUSDT        68.4     0.000000           NaN   -68.400000     清仓
    KAVAUSDT       -181.8     0.000000           NaN   181.800000     清仓

    """
    exchange = account_info['exchange']
    swap_position = account_info['swap_position']
    spot_position = account_info['spot_position']

    swap_position.reset_index(inplace=True)
    swap_position['symbol_type'] = 'swap'

    spot_position.reset_index(inplace=True)
    spot_position['symbol_type'] = 'spot'

    all_position = pd.concat([swap_position, spot_position], ignore_index=True)
    # ===计算目标持仓量
    # ** 由于现货的手续费是扣除在购买的现货中，所以这里下单的时候，对下单金额进行打折
    # ** vol * price * (1 + r) = 下单金额
    # ** vol = 下单金额 / (1 + r) / price
    # select_coin['每个策略单方向分配金额'] = (swap_equity + spot_equity) * leverage * select_coin['cap_weight'] / 2 / 1.001
    # select_coin['每个策略单方向选币数量'] = select_coin.groupby(['candle_begin_time', 'strategy', '方向'])['symbol'].transform('size')
    # select_coin['目标持仓量'] = select_coin['每个策略单方向分配金额'] / select_coin['每个策略单方向选币数量'] / select_coin['close'] * select_coin['方向']

    # ===创建symbol_order，用来记录要下单的币种的信息
    # =创建一个空的symbol_order，里面有select_coin（选中的币）、all_position（当前持仓）中的币种
    symbol_order = pd.concat([select_coin[['symbol', 'symbol_type']], all_position[['symbol', 'symbol_type']]], ignore_index=True)
    symbol_order.drop_duplicates(subset=['symbol', 'symbol_type'], inplace=True)
    symbol_order.set_index(['symbol', 'symbol_type'], inplace=True)
    all_position.set_index(['symbol', 'symbol_type'], inplace=True)
    # =symbol_order中更新当前持仓量
    symbol_order['当前持仓量'] = all_position['当前持仓量']
    symbol_order['当前持仓量'].fillna(value=0, inplace=True)

    # =目前持仓量当中，可能可以多空合并
    if select_coin.empty:
        symbol_order['目标持仓量'] = 0
    else:
        symbol_order['目标持仓量'] = select_coin.groupby(['symbol', 'symbol_type'])[['目标持仓量']].sum()
        symbol_order['目标持仓量'].fillna(value=0, inplace=True)

    # ===计算实际下单量和实际下单资金
    symbol_order['实际下单量'] = symbol_order['目标持仓量'] - symbol_order['当前持仓量']

    # ===计算下单的模式，清仓、建仓、调仓等
    symbol_order = symbol_order[symbol_order['实际下单量'] != 0]  # 过滤掉实际下当量为0的数据
    symbol_order.loc[symbol_order['目标持仓量'] == 0, '交易模式'] = '清仓'
    symbol_order.loc[symbol_order['当前持仓量'] == 0, '交易模式'] = '建仓'
    symbol_order['交易模式'].fillna(value='调仓', inplace=True)  # 增加或者减少原有的持仓，不会降为0

    if select_coin.empty:
        symbol_order['实际下单资金'] = np.nan
    else:
        select_coin.sort_values('candle_begin_time', inplace=True)
        symbol_order['close'] = select_coin.groupby(['symbol', 'symbol_type'])[['close']].last()
        symbol_order['实际下单资金'] = symbol_order['实际下单量'] * symbol_order['close']
        del symbol_order['close']
    symbol_order.reset_index(inplace=True)

    # 补全历史持仓的最新价格信息
    if symbol_order['实际下单资金'].isnull().any():
        symbol_swap_price = fetch_binance_ticker_data(exchange)  # 获取合约的最新价格
        symbol_spot_price = fetch_binance_ticker_data(exchange, symbol_type='spot')  # 获取现货的最新价格

        # 获取合约中实际下单资金为nan的数据
        swap_nan = symbol_order.loc[(symbol_order['实际下单资金'].isnull()) & (symbol_order['symbol_type'] == 'swap')]
        if not swap_nan.empty:
            # 补充一下合约中实际下单资金为nan的币种数据，方便后续进行拆单
            for _index in swap_nan.index:
                symbol_order.loc[_index, '实际下单资金'] = symbol_order.loc[_index, '实际下单量'] * symbol_swap_price[swap_nan.loc[_index, 'symbol']]

        # 获取现货中实际下单资金为nan的数据
        # 有些spot不存在价格，无法直接乘，eg：ethw
        spot_nan = symbol_order.loc[(symbol_order['实际下单资金'].isnull()) & (symbol_order['symbol_type'] == 'spot')]
        if not spot_nan.empty:
            has_price_spot = list(set(spot_nan['symbol'].to_list()) & set(symbol_spot_price.index))  # 筛选有USDT报价的现货
            spot_nan = spot_nan[spot_nan['symbol'].isin(has_price_spot)]  # 过滤掉没有USDT报价的现货，没有报价也表示卖不出去
            if not spot_nan.empty:  # 对含有报价的现货，补充 实际下单资金 数据
                # 补充一下现货中实际下单资金为nan的币种数据，方便后续进行拆单
                for _index in spot_nan.index:
                    symbol_order.loc[_index, '实际下单资金'] = symbol_order.loc[_index, '实际下单量'] * symbol_spot_price[spot_nan.loc[_index, 'symbol']]
            else:  # 对没有报价的现货，设置 实际下单资金 为1，进行容错
                symbol_order.loc[spot_nan.index, '实际下单资金'] = 1

    return symbol_order


def save_select_coin(select_coin, run_time, account_name, max_file_limit=999):
    """
    保存选币数据，最多保留999份文件
    :param select_coin: 保存文件内容
    :param run_time: 当前运行时间
    :param account_name: 账户名称
    :param max_file_limit: 最大限制
    """
    # 获取存储文件位置
    file_path = os.path.join(select_coin_path, f"{account_name}_{run_time.strftime('%Y-%m-%d_%H')}.pkl")
    # 保存文件
    select_coin.to_pickle(file_path)
    # 删除多余的文件
    del_file(select_coin_path, max_file_limit, file_suffix='.pkl')


def del_file(file_path, max_file_limit=999, file_suffix='.pkl'):
    """
    删除多余的文件，最限制max_file_limit
    :param file_path: 文件路径
    :param max_file_limit: 最大限制
    :param file_suffix: 文件后缀
    """
    # ===删除多余的flag文件
    files = [_ for _ in os.listdir(file_path) if _.endswith(file_suffix)]  # 获取select_coin_path目录下所有以.pkl结尾的文件
    # 判断一下当前目录下文件是否过多
    if len(files) > max_file_limit:  # 文件数量超过最大文件数量限制，保留近999个文件，之前的文件全部删除
        print(f'目前文件数量: {len(files)}, 文件超过最大限制: {max_file_limit}，准备删除文件')
        files = sorted(files, reverse=True)  # 文件名称是时间命名的，所以这里倒序排序结果，距离今天时间越近的排在前面，距离距离今天时间越远的排在最后。例：[2023-04-02_08, 2023-04-02_07, 2023-04-02_06···]
        rm_files = files[max_file_limit:]  # 获取需要删除的文件列表

        # 遍历删除文件
        for _ in rm_files:
            os.remove(os.path.join(file_path, _))  # 删除文件
            print(os.path.join(file_path, _), '删除文件完成')


def check_leverage(account_info):
    """
    检查杠杆

    现货模式下，限制杠杆最大值不能超过 1.3
    """
    leverage = account_info['leverage']
    has_spot_strategy = account_info['has_spot_strategy']
    # 获取当前策略信息
    for stg in account_info['strategy_list']:
        if stg['strategy'].startswith('Shift'):  # 轮动策略，这里就不做检查了
            shift_strategy = __import__(f'shift.{stg["strategy"]}', fromlist=('',))
            # 检查持仓是否一致
            hold_period_list = set()
            for _stg_name in list(shift_strategy.base_strategy_dict.keys()):
                strategy = __import__(f'strategy.{_stg_name}', fromlist=('',))
                hold_period_list.add(strategy.hold_period)
            if len(hold_period_list) > 1:
                print('=' * 80)
                print(f'\t轮动策略配置的子策略持仓周期不一致，程序自动退出。')
                print('=' * 80)
                exit()
        else:
            strategy = __import__(f'strategy.{stg["strategy"]}', fromlist=('',))
            long_select_coin_num = strategy.long_select_coin_num  # 获取策略里面的多头选币数量
            short_select_coin_num = strategy.short_select_coin_num  # 获取策略里面的空头选币数量
            is_only_long = long_select_coin_num != 0 and short_select_coin_num == 0  # 纯多头选币
            is_only_short = long_select_coin_num == 0 and short_select_coin_num != 0  # 纯空头选币

            if has_spot_strategy:
                if leverage > 1.3:
                    print('=' * 80)
                    print(f'\t已开启现货模式，当前实盘配置杠杆【{leverage}】,已经超过最大杠杆限制【1.3】，程序自动退出。')
                    print('=' * 80)
                    exit()
                if is_only_long and leverage > 1:
                    print('=' * 80)
                    print(f'\t已开启现货模式，当前实盘配置杠杆【{leverage}】，已经超过纯多头的最大杠杆限制【1】，程序自动退出。')
                    print('=' * 80)
                    exit()
                if is_only_short:
                    print('=' * 80)
                    print(f'\t已开启现货模式，不支持纯空头选币，程序自动退出。')
                    print('=' * 80)
                    exit()
            else:
                if is_only_short and isinstance(short_select_coin_num, str):
                    print('=' * 80)
                    print(f'\t已开启合约模式，纯空头选币仅仅支持选币数量为数字，不支持【long_nums】，程序自动退出。')
                    print('=' * 80)
                    exit()


def calc_spot_need_usdt_amount(account_info, select_coin):
    """
    计算现货账号需要划转多少usdt过去
    """
    # 获取需要计算的数据
    spot_equity = account_info['spot_equity']
    exchange = account_info['exchange']
    wechat_webhook_url = account_info['wechat_webhook_url']
    is_pure_long = account_info['is_pure_long']

    # 现货下单总资金
    spot_strategy_equity = 0 if select_coin.empty else select_coin[select_coin['symbol_type'] == 'spot']['单币下单金额'].sum()

    # 计算现货下单总资金 与 当前现货的资金差值，需要补充（这里是多加2%的滑点）
    diff_equity = spot_strategy_equity * 1.02 - spot_equity

    # 获取合约账户中可以划转的USDT数量
    swap_assets = retry_wrapper(exchange.fapiPrivateV2_get_account, params={'timestamp': ''}, func_name='获取U本位合约账户净值')  # 获取账户净值
    swap_assets = pd.DataFrame(swap_assets['assets'])
    swap_max_withdraw_amount = float(swap_assets[swap_assets['asset'] == 'USDT']['maxWithdrawAmount'])  # 获取可划转USDT数量
    swap_max_withdraw_amount = swap_max_withdraw_amount * 0.99  # 出于安全考虑，给合约账户预留1%的保证金

    # 计算可以划转的USDT数量
    transfer_amount = min(diff_equity, swap_max_withdraw_amount)
    # 现货需要的USDT比可划转金额要大，这里发送信息警告(前提：非纯多现货模式下)
    if not is_pure_long and diff_equity > swap_max_withdraw_amount:
        msg = '======警告======\n\n'
        msg += f'现货所需金额:{diff_equity:.2f}\n'
        msg += f'合约可划转金额:{swap_max_withdraw_amount:.2f}\n'
        msg += '划转资金不足，可能会造成现货下单失败！！！'
        # 重复发送五次
        for i in range(0, 5, 1):
            send_wechat_work_msg(msg, wechat_webhook_url)
            time.sleep(3)

    return transfer_amount


def create_finish_flag(flag_path, run_time, signal):
    """
    创建数据更新成功的标记文件
    如果标记文件过多，会删除7天之前的数据

    :param flag_path:标记文件存放的路径
    :param run_time: 当前的运行是时间
    :param signal: 信号
    """
    # ===判断数据是否完成
    if signal > 0:
        print('当前数据更新出现错误信号: ', signal, '，数据更新没有完成，当前小时不生成 flag 文件')
        return

    # ===生成flag文件
    # 指定生成文件名称
    index_config_path = os.path.join(flag_path, f"{run_time.strftime('%Y-%m-%d_%H')}.flag")  # 例如文件名是：2023-04-02_08.flag
    # 更新信息成功，生成文件
    with open(index_config_path, 'w', encoding='utf-8') as f:
        f.write('更新完成')
        f.close()

    # ===删除多余的flag文件
    del_file(flag_path, 7 * 24, file_suffix='.flag')


def get_account_info(account_info, is_only_spot_account=False, is_operate=True):
    """
    获取账户信息
    :param account_info:            config中配置跌账户
    :param is_only_spot_account:    是否只保留现货交易的账户
    :param is_operate:              是否进行账户的调整操作
    :return:
    """
    # 从账户列表中拿到所有的账户名称
    # 注意点：直接使用account_list.keys()去遍历，del其中任一账户都会报错，使用list暂存一下不会出现问题
    account_list = list(account_info.keys())
    # 遍历账号，获取每个账号下的净值和持仓
    for account_name in account_list:
        try:
            # ===获取账户配置
            account = account_info[account_name]

            # 是否只保留现货账户
            if is_only_spot_account and not account['has_spot_strategy']:  # 如果只保留有现货交易的账户，非现货交易账户被删除
                del account_info[account_name]
                continue

            # ===加载合约和现货的数据
            # =获取U本位合约持仓
            swap_position = get_position_df(account['exchange'])
            # =获取U本位合约账户净值(不包含未实现盈亏)
            swap_equity = get_equity(account['exchange'])

            # ===加载现货交易对的信息
            # 判断是否使用现货实盘
            if account['has_spot_strategy']:  # 如果使用现货实盘，需要读取现货交易对信息和持仓信息
                # =获取现货持仓净值(包含实现盈亏，这是现货自带的)
                spot_equity, spot_position, spot_usdt, dust_spot = get_spot_position_and_equity(account['exchange'])
                if account['if_transfer_bnb']:
                    # =小额资产转换
                    transfer_bnb(account['exchange'], dust_spot)
            else:  # 不使用现货实盘，设置现货价值为默认值0，持仓为空
                spot_equity = 0
                spot_position = pd.DataFrame()
                spot_usdt = 0
            print('合约净值(不含浮动盈亏)：', swap_equity, '\t现货净值:', spot_equity, '\t现货的USDT：', spot_usdt)

            # 判断当前账号是否有资金
            if swap_equity + spot_equity <= 0:
                print(f'账户: {account_name}，资金不足，移除账户配置')
                del account_info[account_name]  # 账户资金不足，移除当前账户
                continue

            # 判断是否需要进行账户的调整（划转，买BNB，调整页面杠杆）
            if is_operate:
                # ===设置一下页面最大杠杆
                reset_leverage(account['exchange'], 5)

                # ===将现货中的U转到合约账户(预留2倍bnb手续费抵扣资金)
                if spot_usdt > account['buy_bnb_value'] * 2:
                    bn_transfer_spot_to_u(account['exchange'], round(spot_usdt - account['buy_bnb_value'] * 2, 1))
                    spot_equity -= round(spot_usdt - account['buy_bnb_value'] * 2, 1)  # 现货总净值扣除掉划走USDT
                    swap_equity += round(spot_usdt - account['buy_bnb_value'] * 2, 1)  # 合约总净值加上掉划走USDT

                # ===补充BNB操作(用于抵扣手续费)
                if account['if_use_bnb_burn']:  # 判断是否开启BNB燃烧
                    # 合约资金有钱补充BNB
                    if swap_equity > account['buy_bnb_value']:
                        all_buy_bnb_value = replenish_bnb(account['exchange'], account['buy_bnb_value'], account['has_spot_strategy'])
                        if all_buy_bnb_value > 0:
                            # 发送信息给机器人
                            send_wechat_work_msg(f'补充BNB抵扣手续费操作完成，补充金额：{all_buy_bnb_value:.4f} U', account_info[account_name]['wechat_webhook_url'])

            # ===更新账户信息
            account_info[account_name]['swap_position'] = swap_position
            account_info[account_name]['swap_equity'] = swap_equity
            account_info[account_name]['spot_position'] = spot_position
            account_info[account_name]['spot_equity'] = spot_equity - account['buy_bnb_value'] * 2  # 保留bnb手续费抵扣资金

            # ===休息一下
            time.sleep(2)
        except KeyboardInterrupt:
            print('退出')
            exit()
        except BaseException as e:
            msg = f'账户: {account_name}，更新账户信息出现问题，移除账户配置'
            print(msg)
            print(traceback.format_exc())
            send_wechat_work_msg(msg, account_info[account_name]['wechat_webhook_url'])
            del account_info[account_name]  # 发生错误，移除当前账户

    return account_info


def refresh_diff_time(exchange):
    """
    刷新本地电脑与交易所的时差
    :param exchange:    交易所对象
    """
    server_time = exchange.fetch_time()  # 获取交易所时间
    diff_timestamp = int(time.time() * 1000) - server_time  # 计算时差
    glob_var.update_diff_time(diff_timestamp)  # 更新到全局变量中


def save_and_merge_select(select_coin, account_name, account_info, run_time, if_rebalance):
    # 解析账户信息
    swap_equity = account_info['swap_equity']
    spot_equity = account_info['spot_equity']
    leverage = account_info['leverage']
    exchange = account_info['exchange']

    # 构建本次存放选币下单的文件
    order_file = os.path.join(order_path, f'{account_name}_order.csv')

    # 杠杆为0，表示清仓
    if leverage == 0:
        select_coin.drop(select_coin.index, inplace=True)
        # 清仓之后删除本地文件
        if os.path.exists(order_file):
            os.remove(order_file)
        return select_coin

    # 计算目标持仓量
    # 如果当前选币为纯多，自动降低下单杠杆5%，防止现货下单滑点的影响（因为现货杠杆不能超过1，所以直接设置0.95）
    dynamic_leverage = 0.95 if (len(select_coin['方向'].unique()) == 1) and (1 in select_coin['方向']) else leverage
    select_coin['每个策略单offset分配金额'] = (swap_equity + spot_equity) * dynamic_leverage * select_coin['cap_weight'] / select_coin['offset_num'] / 1.001
    select_coin['每个策略单offset方向数量'] = select_coin.groupby(['candle_begin_time', 'strategy'])['方向'].transform('nunique')
    select_coin['每个策略各offset方向分配金额'] = select_coin['每个策略单offset分配金额'] / select_coin['每个策略单offset方向数量']
    select_coin['每个策略单方向选币数量'] = select_coin.groupby(['candle_begin_time', 'strategy', '方向'])['symbol'].transform('size')
    select_coin['单币下单金额'] = select_coin['每个策略各offset方向分配金额'] / select_coin['每个策略单方向选币数量']
    select_coin['目标持仓量'] = select_coin['单币下单金额'] / select_coin['close'] * select_coin['方向']

    # 设置指定字段保留
    cols = ['candle_begin_time', '换仓时间', 'symbol', 'symbol_type', 'close', '方向', 'offset', 'strategy', 'cap_weight',
            'offset_num', '每个策略单offset方向数量', '每个策略单方向选币数量', '单币下单金额', '目标持仓量']
    select_coin = select_coin[cols]
    if if_rebalance:
        return select_coin

    # 判断本地是否存在下单文件
    if os.path.exists(order_file):
        # 读取本地文件
        local_order_df = pd.read_csv(order_file, encoding='gbk', parse_dates=['candle_begin_time', '换仓时间'])
        # 获取最新价格
        swap_price = fetch_binance_ticker_data(exchange, symbol_type='swap')
        spot_price = fetch_binance_ticker_data(exchange, symbol_type='spot')
        local_order_df['最新价格'] = local_order_df.apply(lambda row: swap_price[row['symbol']] if row['symbol_type'] == 'swap' else spot_price[row['symbol']], axis=1)

        # 获取加仓资金
        all_equity = (swap_equity + spot_equity) * leverage
        # 当前持仓的资金是多少
        spot_df = local_order_df[local_order_df['symbol_type'] == 'spot']
        swap_df = local_order_df[local_order_df['symbol_type'] == 'swap']
        pos_equity = swap_df['单币下单金额'].sum() + sum(spot_df['目标持仓量'] * spot_df['最新价格'])
        # 加减仓的资金
        dynamic_equity = all_equity - pos_equity
        # 按照策略资金比例分配加仓金额
        local_order_df['加仓金额'] = dynamic_equity * local_order_df['cap_weight']  # 每个策略的加仓金额
        local_order_df['加仓金额'] = local_order_df['加仓金额'] / local_order_df['offset_num']  # 每个offset加仓的金额
        local_order_df['加仓金额'] = local_order_df['加仓金额'] / local_order_df['每个策略单offset方向数量']  # 每个方向的加仓金额
        local_order_df['加仓金额'] = local_order_df['加仓金额'] / local_order_df['每个策略单方向选币数量']  # 每个币的加仓金额

        # 获取当前最新的策略信息
        strategy_list = pd.concat([local_order_df[['strategy', 'offset']], select_coin[['strategy', 'offset']]])
        strategy_list.drop_duplicates(subset=['strategy', 'offset'], keep='last', inplace=True)
        strategy_list.reset_index(inplace=True, drop=True)

        # 按照策略进行遍历
        for index, row in strategy_list.iterrows():
            # 本地当前策略数据
            cond1 = (local_order_df['strategy'] == row['strategy']) & (local_order_df['offset'] == row['offset'])
            _local_df = local_order_df[cond1]
            # 最新选币的策略数据
            cond2 = (select_coin['strategy'] == row['strategy']) & (select_coin['offset'] == row['offset'])
            _select_df = select_coin[cond2]
            # 本地存储数据为空，表示是新增的策略需要按照最新的资金进行分配处理，直接将 _select_df 数据追加到 local_order_df 中
            if _local_df.empty:
                local_order_df = pd.concat([local_order_df, _select_df], ignore_index=True)
                continue

            # 选币数据为空，表示是移除的策略需要按照最新的资金进行分配处理，直接将 _local_df 数据移除 local_order_df
            if _select_df.empty:
                local_order_df = local_order_df.loc[~cond1]
                continue

            # 本地文件数据进入换仓阶段，
            if _local_df.iloc[0]['换仓时间'] + pd.Timedelta(hours=utc_offset) <= run_time:
                # 删除掉本地需要换仓的数据
                local_order_df.drop(_local_df.index, inplace=True)
                # 计算平仓资金，用来进行换仓时使用
                pos_net = (_local_df['close'] + _local_df['方向'] * (_local_df['最新价格'] - _local_df['close'])) * abs(_local_df['目标持仓量'])
                pos_net = sum(pos_net)
                if _local_df.iloc[0]['换仓时间'] + pd.Timedelta(hours=utc_offset) == run_time:
                    pos_net += _local_df['加仓金额'].sum()
                # 根据上周期平仓时仓位价值，来开下周期的仓位
                _select_df['单币下单金额'] = pos_net / _select_df['每个策略单方向选币数量'] / _select_df['每个策略单offset方向数量']
                # 将需要换仓的数据合并过来
                local_order_df = pd.concat([local_order_df, _select_df], ignore_index=True)
            else:
                # 计算当前最新的资金比例
                local_order_df.loc[cond1, 'cap_weight'] = _select_df['cap_weight'].iloc[0]  # 同一个策略下的资金占比是一样的
                # 计算当前最新的单币下单金额
                local_order_df.loc[cond1, '单币下单金额'] = local_order_df.loc[cond1, 'cap_weight'] / np.array(_local_df['cap_weight']) * _local_df['单币下单金额'] + _local_df['加仓金额']

        # 计算最新的目标持仓量
        local_order_df['目标持仓量'] = local_order_df['单币下单金额'] / local_order_df['close'] * local_order_df['方向']

        # 覆盖掉 select_coin
        select_coin = local_order_df.copy()
    else:
        select_coin['加仓金额'] = 0  # 首次默认设置加仓金额为0

    # 保存筛选后的选币文件
    select_coin.reset_index(inplace=True, drop=True)
    select_coin.to_csv(order_file, encoding='gbk', index=False)

    return select_coin


def check_first_run_shift(account_info):
    """
    判断当前账号下的轮动选币文件是否存在，来确定是否是首次启动
    """
    # 当前策略选币文件不存在，则表示首次启动，跳过当前下单（防止首次资金曲线计算时间过长，价格波动影响下单）
    file_path = os.path.join(data_path, f'{account_info["account_name"]}/shift_select_coin_result.pkl')
    cond1 = not os.path.exists(file_path)  # 本地文件不存在轮动选币文件
    cond2 = account_info['strategy_list'][0]['strategy'].startswith('Shift')  # config里面配置的是轮动策略

    return cond1 and cond2
